package za.org.meraka.mu;

import java.io.InputStream;
import java.io.BufferedReader;
import java.io.FileReader;

import java.util.Properties;
import java.util.StringTokenizer;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
import java.util.ListIterator;
import java.util.Date;
import java.lang.Comparable;
import java.util.TreeSet;
import java.util.HashSet;
import java.util.Set;
import java.util.Collections;

public class Mu {

	Properties configuration;
	Properties stopwords;

	//
	// information about the primary topic
	// 
	HashMap<String, Properties> topics;
	HashMap<String, List<NgramResult>> topicNgrams;

	//
	// secondary topic information
	HashMap<String, Properties> subTopics;
	HashMap<String, List<NgramResult>> subTopicNgrams;

	//
	// all topic information
	//
	HashMap<String, Properties> allTopics;

	//
	// linkages between primary topics
	// and secondary topics
	//
	HashMap<String, List<String>> linkages;

	//
	// ngrams to ignore
	//
	List<String> ignores;

	int ngramLength = -1;
  	int majorTopicCnt = -1;
  	int minorTopicCnt = -1;
  	int totalTopicCnt = -1;

	double spellSimilarity = 0.0;

	boolean debug = false;


/*
 * load the configuration properties file
 */
private void loadConfiguration() {
	try {
		configuration = new Properties();
		InputStream is = Mu.class.getResourceAsStream(
			"/mu.properties");
		configuration.load(is);
		ngramLength = Integer.parseInt(configuration.getProperty("ngramlength"));
		majorTopicCnt = Integer.parseInt(configuration.getProperty("majorTopicCnt"));
		minorTopicCnt = Integer.parseInt(configuration.getProperty("minorTopicCnt"));
		totalTopicCnt = Integer.parseInt(configuration.getProperty("totalTopicCnt"));

		spellSimilarity = Double.parseDouble(configuration.getProperty("spellSimilarity"));

		if ( debug ) {
			System.out.println("ngram length is " + ngramLength);
			System.out.println("majorTopicCnt is " + majorTopicCnt);
			System.out.println("minorTopicCnt is " + minorTopicCnt);
			System.out.println("totalTopicCnt is " + totalTopicCnt);
			System.out.println("spell similarity is " + spellSimilarity);
		}

	}
	catch ( Throwable t ) {
		System.out.println("loadConfiguration throws " + t);
	}
}

private Properties loadProperties(String resourceName) {
	try {
		Properties tmp = new Properties();
		InputStream is = Mu.class.getResourceAsStream(
			"/" + resourceName);
		tmp.load(is);
		return tmp;
	}
	catch ( Throwable t ) {
		System.out.println("loadProperties throws " + t);
	}

	return null;
}


/*
 * load the stop words
 */
private void loadStopWords(String filename) {
	try {
		stopwords = loadProperties(filename);
	}
	catch ( Throwable t ) {
		System.out.println("loadStopWords throws " + t);
	}
}

/*
 * process the ignores fields
 */
private String ignores(String input) {
	//
	// need to ignore some stuff
	//
	if ( ignores.size() > 0 ) {
		String words = input;
		//System.out.println("words are     " + words);
		ListIterator<String> iterator = ignores.listIterator();
		while ( iterator.hasNext() ) {
			String remove = iterator.next();
			//System.out.println("ignoring " + remove);
			words = words.replaceAll(remove, " ");
		}
		//System.out.println("words are now " + words);
		return words;
	}
	else
		return input;
}

/*
 * load the various topic data
 */
private void loadTopics(String prefix, HashMap<String, Properties> topics, HashMap<String, List<NgramResult>> ngrams) {
	try {
		Enumeration enumTopics = configuration.propertyNames();
	
		while ( enumTopics.hasMoreElements() ) {
			String propertyName = (String)enumTopics.nextElement();
			if ( propertyName.startsWith(prefix) ) {
				int pos = propertyName.indexOf("_");
				String topicName = propertyName.substring(pos + 1);
				String resourceName=configuration.getProperty(propertyName);
				Properties tmp = loadProperties(resourceName);
				topics.put(topicName, tmp);
				allTopics.put(topicName, tmp);
				Enumeration enumKeys = tmp.keys();
				StringBuffer topicString = new StringBuffer();
				while ( enumKeys.hasMoreElements() ) {
					topicString.append(enumKeys.nextElement() + " ");
				}

				List<NgramResult> ngram = null;
				ngram = Ngram.processString(ignores(topicString.toString()), ngramLength);
				ngrams.put(topicName, ngram);
			}
		}

		/*
		Iterator<String> iterator = topics.keySet().iterator();
		while ( iterator.hasNext() ) {
			String topicName = iterator.next();
			Properties tmp = topics.get(topicName);
		}
		*/
	}
	catch ( Throwable t ) {
		System.out.println("loadTopics throws " + t);
	}
}

private void loadMainTopics() {
	try {
		allTopics = new HashMap<String, Properties>();
		topics = new HashMap<String, Properties>();
		topicNgrams = new HashMap<String, List<NgramResult>>();
		loadTopics("topic_", topics, topicNgrams);
	}
	catch ( Throwable t ) {
		System.out.println("loadMainTopics throws " + t);
	}
}

private void loadSubTopics() {
	try {
		subTopics = new HashMap<String, Properties>();
		subTopicNgrams = new HashMap<String, List<NgramResult>>();
		loadTopics("subtopic_", subTopics, subTopicNgrams);
		if ( debug )
			System.out.println("subTopicNgrams = " + subTopicNgrams.keySet());
	}
	catch ( Throwable t ) {
		System.out.println("loadSubTopics throws " + t);
	}
}

private void loadLinkages() {
	linkages = new HashMap<String, List<String>>();
	Iterator<String> iterator = topics.keySet().iterator();
	while ( iterator.hasNext() ) {
		String topicName = iterator.next();
		String tmp = configuration.getProperty(topicName);
		if ( tmp != null ) {
			List<String> links = new ArrayList<String>();
			StringTokenizer toks = new StringTokenizer(tmp, ",");
			while (toks.hasMoreTokens() ) {
				String st = toks.nextToken();
				links.add(st);
			}
			if ( debug )
				System.out.println("topic " + topicName + " has " + links.size() + " linkages " + links);
			linkages.put(topicName, links);
	
		}
	}
}

private void loadIgnores() {
	try {
		Enumeration enumTopics = configuration.propertyNames();

		ignores = new ArrayList<String>();
	
		while ( enumTopics.hasMoreElements() ) {
			String propertyName = (String)enumTopics.nextElement();
			if ( propertyName.startsWith("ignore_") ) {
				String stringToIgnore=configuration.getProperty(propertyName).replaceAll("_", " ");
				if ( debug )
					System.out.println("need to ignore [" + stringToIgnore + "]");
				ignores.add(stringToIgnore);
			}
		}
	}
	catch ( Throwable t ) {
		System.out.println("problem with ignores " + t);
	}
}

private void init(boolean debug) {
	try {
		this.debug = debug;
		loadConfiguration();
		loadIgnores();
		loadStopWords(configuration.getProperty("stopwords"));
		loadMainTopics();
		loadSubTopics();
		loadLinkages();
	}
	catch ( Throwable t ) {
		System.out.println("init throws " + t);
	}
}

public Mu(boolean debug) {
	init(debug);
}

public Mu() {
	init(false);
}

/*
 * remove the stop words from a String
 * and stem the result and words of just
 * one character length
 */
public String removeStopWords(String input) {
	String separators = " []{}0123456789\\.,<>/?;:!@#$%^&*()_-=+:;-`'\\\"";
	StringTokenizer toks = new StringTokenizer(input, separators);

	StringBuffer returnValue = new StringBuffer();
	while (toks.hasMoreTokens() ) {
		String tok = toks.nextToken();
		if ( tok.length() < 2 ) {
			continue;
		}
		String test = stopwords.getProperty(tok);
		if ( test == null ) {
			String stemmed = MxitUtilities.stem(tok);
			returnValue.append(stemmed + " ");
		}
	}
	
	return returnValue.toString();
}

/*
 * extract just the math terms out of the terms
 * remaining after the stop words have been
 * removed
 */
public String extractMathTerms(String input) {

	//
	// input should be space delimited by now
	//
	StringTokenizer toks = new StringTokenizer(input, " ");

	StringBuffer returnValue = new StringBuffer();

	List<String> keySet = new ArrayList<String>();
	keySet.addAll(topics.keySet());
	keySet.addAll(subTopics.keySet());

	//
	// loop on words
	//
	while (toks.hasMoreTokens() ) {
		String tok = toks.nextToken();
		if ( debug )
			System.out.println("tok " + tok);

		//
		// this is done just in case we need to
		// look for a good match
		//
		List<NgramResult> tokNgram = Ngram.processString(
			ignores(MxitUtilities.stem(tok)), ngramLength);
		

		//
		// loop on installed topics
		//
		Iterator<String> iterator = keySet.iterator(); //topics.keySet().iterator();
		boolean exactHit = false;
		while ( iterator.hasNext() ) {
			String topicName = iterator.next();
			Properties tmp = allTopics.get(topicName);
			//Properties tmp = topics.get(topicName);

			//
			// look for exact match in topic properties
			//
			String ret = tmp.getProperty(tok);
			//
			// found a stemmed math term that matches
			//
			if ( ret != null ) {
				exactHit = true;
				returnValue.append(tok + " ");
				break;
			}
		}


		//
		// if we do not have an exact hit,
		// then lets use some Ngrams to get
		// the next best hit....
		//
		if ( ! exactHit ) {
			iterator = keySet.iterator(); //topics.keySet().iterator();
			List<Score> possibleWords = new ArrayList<Score>();
			while ( iterator.hasNext() ) {
				String topicName = iterator.next();
				Properties tmp = allTopics.get(topicName);
				//Properties tmp = topics.get(topicName);
	
				//
				// we should do an ngram comparision
				// on each word to find a 90% or above
				// match - that would just mean a spelling
				// error
				//
				Enumeration mathTermEnumeration = tmp.propertyNames();
				String bestWord = new String();
				int bestSimilarity = 0;
				while ( mathTermEnumeration.hasMoreElements() ) {
					String mathTerm = (String)mathTermEnumeration.nextElement();
					List<NgramResult> mathTermNgram = Ngram.processString(
						ignores(MxitUtilities.stem(mathTerm)), ngramLength);
        				int c = Ngram.common(2, mathTermNgram, tokNgram);
        				int u = Ngram.union(2, mathTermNgram, tokNgram);
        				double sim = (double)c/(double)u;
					int similarity = (new Double(sim * 1000)).intValue();
					if ( bestSimilarity < similarity ) {
						bestSimilarity = similarity;
						bestWord = mathTerm;
					}
					//
					// we are just guess with this cut off value right now
					//
					if ( sim >= spellSimilarity ) {
						Score score = new Score();
						score.setTopic(bestWord);
						score.setSimilarity(bestSimilarity);
						possibleWords.add(score);
					}
					if ( debug ) {
						if ( similarity > 0 ) {
							System.out.print("token " + tok);
							System.out.print("\tmathterm " + mathTerm);
							System.out.print("\tcommon is " + c);
							System.out.print("\tunion is " + u);
							System.out.println("similarity is " + similarity);
						}
					}
				}
			}
			if ( possibleWords.size() > 0 ) {
				Collections.sort(possibleWords);
				returnValue.append(possibleWords.get(0).getTopic()  + " ");
				if ( debug )
					System.out.println("best is really " + returnValue);
			}
		}
	}
	
	return returnValue.toString();
}

private String readTestFile(String filename) {
	StringBuffer conversation = new StringBuffer();

	try {
		BufferedReader br = new BufferedReader( new FileReader(filename) ) ;
		String line;
		while ( (line = br.readLine() ) != null ) {
			//System.out.println("read - " + line);
			//conversation.append(line + "|");
			conversation.append(line + " ");
		}
	}
	catch ( Throwable t ) {
		System.out.println("readTestFile throwing " + t);
	}

	return conversation.toString();
}

private List<Score> determineTopic(String input, Iterator<String> iterator, HashMap<String, List<NgramResult>> ngrams, int topicCnt) {
	try {
		List<Score> list = new ArrayList<Score>();

		List<NgramResult> inputNgram = Ngram.processString(ignores(input), ngramLength);
		while ( iterator.hasNext() ) {
			String topicName = iterator.next();
			if ( debug )
				System.out.println("testing " + topicName);
			List<NgramResult> tmp = ngrams.get(topicName);
			if ( debug && tmp == null )
				System.out.println("tmp == null for topicName " + topicName);
        		int c = Ngram.common(1, tmp, inputNgram);
        		int u = Ngram.union(1, tmp, inputNgram);
        		double sim = (double)c/(double)u;
			int similarity = (new Double(sim * 1000)).intValue();
			Date second = new Date();
			Score score = new Score();
			score.setTopic(topicName);
			score.setSimilarity(similarity);
			if ( score.getSimilarity() != 0 ) {
				list.add(score);
			}
		}
	
		Collections.sort(list);
		int i;

		if ( debug ) {
			/*
			for(i=0; i<list.size(); i++)  {
				Score s = list.get(i);
				System.out.println("@@@ " + s.getTopic() + " " + s.getSimilarity());
			}
			*/
		}

		List<Score> top = new ArrayList();
		int score = -1;
		for(i=0; i<list.size(); i++)  {
			if ( debug)
			System.out.println("i= " + i + " topicCnt= " + topicCnt + 
				"list.get(i).getSimilarity= " + list.get(i).getSimilarity() +
				"list.get(topicCnt-1).getSimilarity= " +
				list.get(topicCnt-1).getSimilarity());
			if ( i >= topicCnt && list.get(i).getSimilarity() != list.get(topicCnt-1).getSimilarity() ) {
				if ( debug )
					System.out.println("**breaking**");
				break;
			}
			top.add(list.get(i));
			score = list.get(i).getSimilarity();
		}
		return top;
	}
	catch ( Throwable t ) {
		System.out.println("determineTopic throwing " + t);
	}

	return new ArrayList<Score>();
}

private List<Score> determineMainTopic(String input) {
	return determineTopic(input, topics.keySet().iterator(), topicNgrams, majorTopicCnt);
}

private List<Score> determineSubTopic(String input, String mainTopic) {
	/*
	List<String> subTopics = linkages.get(mainTopic);
	Iterator<String> iterator = topics.keySet().iterator();
	if ( subTopics != null && subTopics.size() > 0 ) {
		return determineTopic(input, subTopics.iterator(), subTopicNgrams, minorTopicCnt);
	}
	else {
		return new ArrayList<Score>();
	}
	*/
	Iterator<String> iterator = subTopics.keySet().iterator();
	return determineTopic(input, iterator, subTopicNgrams, minorTopicCnt);
}

private List<Score> determineAllTopics(String input) {
	List<Score> allMainTopics = determineMainTopic(input);
	//List<Score> allSubTopicScores = determineSubTopic(input, "");
	List<Score> allSubTopicScores = new ArrayList<Score>();
	int i;
	for(i=0 ; i < allMainTopics.size(); i++) {
		Score score = allMainTopics.get(i);
		if ( debug )
			System.out.println("topic " + score.getTopic() +
				" " + score.getSimilarity());
		List<Score> subTopicScores = determineSubTopic(input, 
			score.getTopic());
		int j;
		for(j=0; j<subTopicScores.size(); j++) {
			Score subTopicScore = subTopicScores.get(j);
			if ( debug )
				System.out.println("\tsubtopic " +
					subTopicScore.getTopic() + " " +
					subTopicScore.getSimilarity());
		}
		allSubTopicScores.addAll(subTopicScores);
	}

	//
	// do not add duplicates
	//
	HashMap<String, Score> map = new HashMap<String, Score>();
	ListIterator<Score> iterator = allMainTopics.listIterator();
	while ( iterator.hasNext() ) {
		Score score = iterator.next();
		map.put(score.getTopic(), score);
	}
	iterator = allSubTopicScores.listIterator();
	while ( iterator.hasNext() ) {
		Score score = iterator.next();
		map.put(score.getTopic(), score);
	}

	List<Score> list = new ArrayList(map.values());
	Collections.sort(list);
	if ( debug ) {
		ListIterator<Score> tmp = list.listIterator();
		while ( tmp.hasNext() ) {
			Score s = tmp.next();
			System.out.println("** " + s.getTopic() + " " + s.getSimilarity());
		}
	}
	return list;
}

public List<Score> processMessage(String input ) {
	String tmp;
	tmp = removeStopWords(input);
	if ( debug )
		System.out.println("@@ important words: " + tmp);
	tmp = extractMathTerms(tmp);
	if ( debug )
		System.out.println("@@ math terms: " + tmp);
	List<Score> list = determineAllTopics(tmp);
	return list;
}

public String getTopTopics(String input) {
	List<Score> list = processMessage(input);
	StringBuffer st = new StringBuffer();
	int cnt = 0;
	Iterator<Score> iterator = list.iterator();
	while ( iterator.hasNext() ) {
		Score score = iterator.next();
		if ( debug )
			System.out.println("topic " + score.getTopic() + " " + score.getSimilarity());
		st.append(score.getTopic());
		cnt++;
		if ( cnt == totalTopicCnt ) {
			/*
			if ( debug )
				System.out.println("cnt " + cnt + " breaking");
			*/
			break;
		}
		else {
			/*
			if ( debug )
				System.out.println("cnt " + cnt + " continuing");
			*/
			st.append(", ");
		}
	}
	return st.toString();
}

public static void main(String args[]) {
	try {
		String files[] = null;
		Mu mu = null;
		if ( args.length == 0 ) {
			files = new String[1];
			files[0] = "test.txt";
			mu = new Mu(true);
			mu.debug = true;
		}
		else {
			mu = new Mu(false);
			files = args;
		}


		int i;
		for(i=0; i<files.length; i++) {
			String test = mu.readTestFile(files[i]);

			/*
			StringTokenizer toks = new StringTokenizer(test, "|");
			while ( toks.hasMoreTokens() ) {
				String st = toks.nextToken();
				System.out.println("@@ *********** " + st);
				TreeSet<Score> scores = mu.processMessage(st);
				*/
				List<Score> scores = mu.processMessage(test);
				Iterator<Score> iterator = scores.iterator();
				if ( mu.debug )
					System.out.println("*******************final output ");
				while ( iterator.hasNext() ) {
					Score score = iterator.next();
					//System.out.println(score.getTopic() + " " + score.getSimilarity());
				}

				String topTopics = mu.getTopTopics(test);
			
				//if ( mu.debug)
					System.out.println(files[i] + " " + topTopics);
			/*
			}
			*/
			
		}
	}
	catch ( Throwable t ) {
		System.out.println("mu throwing " + t);
	}
}

}

